import requests
import json
import re
import os
from utils import get_book_url


invalid_path = './invalid_urls.txt'
invalid_path2 = './invalid_urls_manual.txt'
valid_path = './valid_urls.txt'


def status_error(status_code):
    return status_code == 0 or status_code >= 400


def test_url_status(url, use_proxy=False):
    timeout = 1
    try:
        if use_proxy:
            res = requests.get(url, timeout=timeout, proxies={
                'http': 'http://192.168.100.1:7890', 'https': 'http://192.168.100.1:7890'})
        else:
            res = requests.get(url, timeout=timeout)
        status_code = res.status_code
    except requests.exceptions.ConnectTimeout:
        return 0
    except requests.exceptions.ReadTimeout:
        return 0
    except requests.exceptions.HTTPError:
        return 0
    except requests.exceptions.SSLError:
        return 0
    except requests.exceptions.ConnectionError:
        return 0
    except requests.exceptions.InvalidURL:
        return 0
    except requests.exceptions.MissingSchema:
        return 0
    except requests.exceptions.ChunkedEncodingError:
        return 0
    except requests.exceptions.TooManyRedirects:
        return 0

    return status_code


def filter_url_valid(src_books: list):
    invalid_list = []
    with open(invalid_path, 'r', encoding='utf-8') as f:
        for line in f:
            invalid_list.append(line.strip().split(' ')[0])
    with open(invalid_path2, 'r', encoding='utf-8') as f:
        for line in f:
            invalid_list.append(line.strip().split(' ')[0])

    valid_list = []
    with open(valid_path, 'r', encoding='utf-8') as f:
        for line in f:
            valid_list.append(line.strip().split(' ')[0])

    print("Book number before filter: ", len(src_books))
    dst_books = []
    for idx in range(len(src_books)):
        book = src_books[idx]
        whole_url = book['bookSourceUrl']
        print(idx, whole_url)
        if not whole_url:
            continue

        main_url = get_book_url(book)

        if main_url in invalid_list:
            pass
        elif main_url in valid_list:
            dst_books.append(book)
        else:
            url_status = test_url_status(main_url)
            if url_status == 0:
                url_status = test_url_status(main_url, True)
            if not status_error(url_status):
                dst_books.append(book)

                valid_list.append(main_url)
                with open(valid_path, 'a', encoding='utf-8') as f:
                    f.write('{} {}\n'.format(main_url, url_status))
            else:
                invalid_list.append(main_url)
                with open(invalid_path, 'a', encoding='utf-8') as f:
                    f.write('{} {}\n'.format(main_url, url_status))

    # with open(invalid_path, 'w') as f:
    #     for url in invalid_list:
    #         f.write(url + '\n')

    print("Book number after filter: ", len(dst_books))
    return dst_books


def main():
    src_dir = './input'
    dst_path = './output/url_ok.json'

    src_books = []
    for file_name in os.listdir(src_dir):
        if file_name.endswith('.json'):
            src_path = os.path.join(src_dir, file_name).replace('\\', '/')
            with open(src_path, 'r', encoding='utf-8') as f:
                cur_books = json.load(f)
            src_books += cur_books

    dst_books = filter_url_valid(src_books)

    with open(dst_path, 'w', encoding='utf-8') as f:
        json_str = json.dumps(dst_books)
        f.write(json_str)


if __name__ == '__main__':
    main()
